package net.trustie.webmagic.one;

import net.trustie.webmagic.utils.SubSite;

import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.HashMap;
import java.util.InvalidPropertiesFormatException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Queue;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

public class Configure {
	private int sleepTimeThread = 5000;
	
	//--长时间睡眠时候的睡眠时间的上下限 -->
    private int minlongSleepTime = 1800000;
    private int maxlongSleepTime = 7200000;
    
    //长时间睡眠时候的睡眠间隔的上下限 -->
    private int minlongSleepInterval = 3600000;
    private int maxlongSleepInterval = 7200000;
	//用以设置请求间隔的两个阈值以产生随机数  (毫秒）
	private int minInterval = 10000;
	private int maxInterval = 20000;
	
	private int retryTimes = 0;
	private int cycleRetryTimes = 0;
	private int timeOut = 5000;
	private String userAgent = "UserAgent.Browser";
	private String charset;
	private int threadNum = 5;
	private boolean isSpawnUrl = true;
	private int startPageIndex = 1;
	private int endInPageIndex = 6;
	private int pageF = 1;
	private int sleepTimeSpider = 36000;
	private String domain;
	private String startUrl;
	private String urlList;
	private String urlPost;
	private int offset;
	private int limitLine;
	private int noUrlWaitTime;
	private String fixAllRelativeHrefs;
	private Properties prop = new Properties();
	private String prefix;
	private String postfix;
	private String extractMethod;
	private String proxyIp;
	private String proxyPort;
	private int mode;
	private int incrementalPages;
	private int incrementSleepTime;
	//private List<SubSite> subSites = new LinkedList<SubSite>();
	private Queue<SubSite> subSites = new LinkedList<SubSite>();
	Logger log4j = Logger.getLogger(Configure.class);
	Document doc = null;
	public Configure() {
		try {
			FileInputStream fis = null;
			fis = new FileInputStream("spider.conf.xml");
			prop.loadFromXML(fis);
			prop.list(System.out);
		} catch (FileNotFoundException e) {
			e.printStackTrace();

		} catch (InvalidPropertiesFormatException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		config();
	}

	public static void main(String args[]){
		Configure conf = new Configure("spider_test");
	}
	public Configure(String confPath) {
		/*confPath = confPath + ".xml";
		try {
			URL url = ClassLoader.getSystemResource(confPath);
			InputStream is = url.openStream();
			prop.loadFromXML(is);
			// prop.list(System.out);
		} catch (FileNotFoundException e) {
			log4j.error("FileNotFoundException & loading default config xml! & no default config file");
			e.printStackTrace();
		} catch (InvalidPropertiesFormatException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		config();*/
		confPath = confPath + ".xml";
		URL url = ClassLoader.getSystemResource(confPath);
		InputStream is;
		try {
			is = url.openStream();
			DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
			DocumentBuilder db = dbf.newDocumentBuilder();  
			doc = db.parse(is); 
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (ParserConfigurationException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (SAXException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
        config();
	}

	
	private void config() {
		
		//star modify----------------------------
		Element root = doc.getDocumentElement();
		//获取站点
		NodeList sites  = ((Element)root.getElementsByTagName("subSites").item(0)).getElementsByTagName("subSite");
		NodeList properties = ((Element)root.getElementsByTagName("properties").item(0)).getChildNodes();
		for(int i =0; i < sites.getLength(); i++){
			//System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
			Node subSite= sites.item(i);
			addSubSite(subSite,subSites);
		}	
		//用map存储properties ，然后再赋值；
		Map<String,String> propertiesMap = new HashMap<String,String>();
		for(int i =0; i < properties.getLength(); i++){
			//System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
			propertiesMap.put(properties.item(i).getNodeName(), properties.item(i).getTextContent());	
		}
		this.domain = propertiesMap.get("domain");
		this.minInterval = Integer.parseInt(propertiesMap.get("minInterval")) ;
		this.maxInterval = Integer.parseInt(propertiesMap.get("maxInterval")) ;
		
		this.minlongSleepTime = Integer.parseInt(propertiesMap.get("minlongSleepTime")) ;
		this.maxlongSleepTime = Integer.parseInt(propertiesMap.get("maxlongSleepTime")) ;
		
		this.minlongSleepInterval = Integer.parseInt(propertiesMap.get("minlongSleepInterval")) ;
		this.maxlongSleepInterval= Integer.parseInt(propertiesMap.get("maxlongSleepInterval")) ;
		//System.out.println(propertiesMap);		this.retryTimes = 5;
		this.timeOut = 20000;
		this.sleepTimeThread = 0;
		this.cycleRetryTimes = 5;
		this.pageF = 1;
		this.threadNum = 1;
	    this.isSpawnUrl = true;
	    this.offset = 0;
	    this.limitLine = 10;
	    this.noUrlWaitTime = 10000;
	    this.extractMethod = "CSSPath";
	    this.proxyIp = "192.168.245.1";
	    this.proxyPort = "3128";
	    this.userAgent = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
	    
	    //目前url修正和信息页抽取共用一个模板
	    this.urlPost = this.subSites.peek().getUrlPost();
	    this.fixAllRelativeHrefs = this.subSites.peek().getFixAllRelativeHrefs();
	   //end of star modify-------------------------
	    
		/*this.sleepTimeThread = Integer.parseInt(prop
				.getProperty("sleepTimeThread"));
		
				
		this.timeOut = Integer.parseInt(prop.getProperty("timeOut"));
		this.retryTimes = Integer.parseInt(prop.getProperty("retryTimes"));
		this.cycleRetryTimes = Integer.parseInt(prop
				.getProperty("cycleRetryTimes"));
		this.threadNum = Integer.parseInt(prop.getProperty("threadNum"));
		this.startPageIndex = Integer.parseInt(prop
				.getProperty("startPageIndex"));
		this.endInPageIndex = Integer.parseInt(prop
				.getProperty("endInPageIndex"));
		this.pageF = Integer.parseInt(prop.getProperty("pageF"));
		this.sleepTimeSpider = Integer.parseInt(prop
				.getProperty("sleepTimeSpider"));
		this.isSpawnUrl = Boolean.parseBoolean(prop.getProperty("isSpawnUrl"));
		this.userAgent = prop.getProperty("userAgent");
		this.domain = prop.getProperty("domain");
		this.startUrl = prop.getProperty("startUrl");
		this.urlList = prop.getProperty("urlList");
		this.urlPost = prop.getProperty("urlPost");
		this.offset = Integer.parseInt(prop.getProperty("offset"));
		this.limitLine = Integer.parseInt(prop.getProperty("limitLine"));
		this.noUrlWaitTime = Integer
				.parseInt(prop.getProperty("noUrlWaitTime"));

		this.fixAllRelativeHrefs = prop.getProperty("fixAllRelativeHrefs");
		this.prefix = prop.getProperty("prefixUrl");
		this.postfix = prop.getProperty("postfixUrl");
		this.extractMethod = prop.getProperty("extractMethod");
		this.proxyIp = prop.getProperty("proxyIp");
		this.proxyPort =prop.getProperty("proxyPort");
		this.mode = Integer.parseInt(prop.getProperty("mode"));
		this.incrementalPages = Integer.parseInt(prop.getProperty("incrementalPages"));
		this.incrementSleepTime = Integer.parseInt(prop.getProperty("incrementSleepTime"));
		
		this.minInterval = Integer.parseInt(prop.getProperty("minInterval"));
		this.maxInterval = Integer.parseInt(prop.getProperty("maxInterval"));*/
	}
	
	

	private void addSubSite(Node site, Queue<SubSite> subSites2) {
		NodeList sitePros = site.getChildNodes();
		Map<String,String> siteProMap = new HashMap<String,String>();
		for(int i =0; i < sitePros.getLength(); i++){
			//System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
			 siteProMap.put(sitePros.item(i).getNodeName(), sitePros.item(i).getTextContent());
		}
		
		//System.out.println(siteProMap);
		SubSite subSite = new SubSite();
		
		subSite.setName(siteProMap.get("name"));
		subSite.setStartPageIndex(Integer.parseInt(siteProMap.get("startPageIndex")));
		subSite.setEndInPageIndex(Integer.parseInt(siteProMap.get("endInPageIndex")));
		subSite.setFixAllRelativeHrefs(siteProMap.get("fixAllRelativeHrefs"));
		subSite.setPrefixUrl(siteProMap.get("prefixUrl"));
		subSite.setPostfixUrl(siteProMap.get("postfixUrl"));
		subSite.setUrlPost(siteProMap.get("urlPost"));
		subSite.setIncrementalPages(Integer.parseInt(siteProMap.get("incrementalPages")));
		subSite.setIncrementSleepTime(Integer.parseInt(siteProMap.get("incrementSleepTime")));
		
		subSites2.offer(subSite);
		//System.out.println(subSite.getStartPageIndex());
	}

	/**
	 * @return the incrementSleepTime
	 */
	public int getIncrementSleepTime() {
		return incrementSleepTime;
	}

	/**
	 * @param incrementSleepTime the incrementSleepTime to set
	 */
	public void setIncrementSleepTime(int incrementSleepTime) {
		this.incrementSleepTime = incrementSleepTime;
	}

	public int getSleepTimeThread() {
		return sleepTimeThread;
	}

	public void setSleepTimeThread(int sleepTimeThread) {
		this.sleepTimeThread = sleepTimeThread;
	}

	public int getRetryTimes() {
		return retryTimes;
	}

	public void setRetryTimes(int retryTimes) {
		this.retryTimes = retryTimes;
	}

	public int getCycleRetryTimes() {
		return cycleRetryTimes;
	}

	public void setCycleRetryTimes(int cycleRetryTimes) {
		this.cycleRetryTimes = cycleRetryTimes;
	}

	public int getTimeOut() {
		return timeOut;
	}

	public void setTimeOut(int timeOut) {
		this.timeOut = timeOut;
	}

	public String getUserAgent() {
		return userAgent;
	}

	public void setUserAgent(String userAgent) {
		this.userAgent = userAgent;
	}

	public String getCharset() {
		return charset;
	}

	public void setCharset(String charset) {
		this.charset = charset;
	}

	public int getThreadNum() {
		return threadNum;
	}

	public void setThreadNum(int threadNum) {
		this.threadNum = threadNum;
	}

	public boolean isSpawnUrl() {
		return isSpawnUrl;
	}

	public void setSpawnUrl(boolean isSpawnUrl) {
		this.isSpawnUrl = isSpawnUrl;
	}

	public int getStartPageIndex() {
		return startPageIndex;
	}

	public void setStartPageIndex(int startPageIndex) {
		this.startPageIndex = startPageIndex;
	}

	public int getEndInPageIndex() {
		return endInPageIndex;
	}

	public void setEndInPageIndex(int endInPageIndex) {
		this.endInPageIndex = endInPageIndex;
	}

	public int getPageF() {
		return pageF;
	}

	public void setPageF(int pageF) {
		this.pageF = pageF;
	}

	public int getSleepTimeSpider() {
		return sleepTimeSpider;
	}

	public void setSleepTimeSpider(int sleepTimeSpider) {
		this.sleepTimeSpider = sleepTimeSpider;
	}

	public String getDomain() {
		return domain;
	}

	public void setDomain(String domain) {
		this.domain = domain;
	}

	public String getStartUrl() {
		return startUrl;
	}

	public void setStartUrl(String startUrl) {
		this.startUrl = startUrl;
	}

	public String getUrlList() {
		return urlList;
	}

	public void setUrlList(String urlList) {
		this.urlList = urlList;
	}

	public String getUrlPost() {
		return urlPost;
	}

	public void setUrlPost(String urlPost) {
		this.urlPost = urlPost;
	}

	public Properties getProp() {
		return prop;
	}

	public void setProp(Properties prop) {
		this.prop = prop;
	}

	public Logger getLog4j() {
		return log4j;
	}

	public void setLog4j(Logger log4j) {
		this.log4j = log4j;
	}

	public int getOffset() {
		return offset;
	}

	public void setOffset(int offset) {
		this.offset = offset;
	}

	public int getLimitLine() {
		return limitLine;
	}

	public void setLimitLine(int limitLine) {
		this.limitLine = limitLine;
	}

	public int getNoUrlWaitTime() {
		return noUrlWaitTime;
	}

	public void setNoUrlWaitTime(int noUrlWaitTime) {
		this.noUrlWaitTime = noUrlWaitTime;
	}

	


	public String getFixAllRelativeHrefs() {
		return fixAllRelativeHrefs;
	}

	public void setFixAllRelativeHrefs(String fixAllRelativeHrefs) {
		this.fixAllRelativeHrefs = fixAllRelativeHrefs;
	}

	public String getPrefix() {
		return prefix;
	}

	public void setPrefix(String prefix) {
		this.prefix = prefix;
	}

	public String getPostfix() {
		return postfix;
	}

	public void setPostfix(String postfix) {
		this.postfix = postfix;
	}

	public String getExtractMethod() {
		return extractMethod;
	}

	public void setExtractMethod(String extraMethod) {
		this.extractMethod = extraMethod;
	}


	/**
	 * @return the proxyIp
	 */
	public String getProxyIp() {
		return proxyIp;
	}

	/**
	 * @return the proxyPort
	 */
	public String getProxyPort() {
		return proxyPort;
	}

	/**
	 * @param proxyIp the proxyIp to set
	 */
	public void setProxyIp(String proxyIp) {
		this.proxyIp = proxyIp;
	}

	/**
	 * @param proxyPort the proxyPort to set
	 */
	public void setProxyPort(String proxyPort) {
		this.proxyPort = proxyPort;
	}

	/**
	 * @return the mode
	 */
	public int getMode() {
		return mode;
	}

	/**
	 * @param mode the mode to set
	 */
	public void setMode(int mode) {
		this.mode = mode;
	}

	/**
	 * @return the incrementalPages
	 */
	public int getIncrementalPages() {
		return incrementalPages;
	}

	/**
	 * @param incrementalPages the incrementalPages to set
	 */
	public void setIncrementalPages(int incrementalPages) {
		this.incrementalPages = incrementalPages;
	}

	

	//每次请求的时间间隔
	public int getMinInterval() {
		return minInterval;
	}

	public void setMinInterval(int minInterval) {
		this.minInterval = minInterval;
	}

	public int getMaxInterval() {
		return maxInterval;
	}

	public void setMaxInterval(int maxInterval) {
		this.maxInterval = maxInterval;
	}

	public int getMinlongSleepTime() {
		return minlongSleepTime;
	}

	public void setMinlongSleepTime(int minlongSleepTime) {
		this.minlongSleepTime = minlongSleepTime;
	}

	public int getMaxlongSleepTime() {
		return maxlongSleepTime;
	}

	public void setMaxlongSleepTime(int maxlongSleepTime) {
		this.maxlongSleepTime = maxlongSleepTime;
	}

	public int getMinlongSleepInterval() {
		return minlongSleepInterval;
	}

	public void setMinlongSleepInterval(int minlongSleepInterval) {
		this.minlongSleepInterval = minlongSleepInterval;
	}

	public int getMaxlongSleepInterval() {
		return maxlongSleepInterval;
	}

	public void setMaxlongSleepInterval(int maxlongSleepInterval) {
		this.maxlongSleepInterval = maxlongSleepInterval;
	}

	public int getSubSiteNum() {
		return this.subSites.size();
	}


	// 获取下一个要处理的站点，并更新当前状态
	public SubSite nexSite() {
		SubSite subSite = this.subSites.poll();
		this.subSites.offer( subSite);
		
		this.startPageIndex  = subSite.getStartPageIndex();
		this.endInPageIndex = subSite.getEndInPageIndex();
		this.fixAllRelativeHrefs = subSite.getFixAllRelativeHrefs();
		this.prefix = subSite.getPrefixUrl();
		this.postfix = subSite.getPostfixUrl();
		this.urlPost = subSite.getUrlPost();
		this.incrementalPages = subSite.getIncrementalPages();
		this.incrementSleepTime = subSite.getIncrementSleepTime();
		
		
		return subSite;
	}
	
	
	
}
